  
;;########################################################################
;; misdsd3.lsp
;; Copyright (c) 1998 by Pedro Valero (valerop@uv.es)
;; Code for Visualization of Missing data 
;; This code is very messy but not bad for being my first experience 
;; with the Lisp Object System. It has to be improved.
;;########################################################################


(defmeth missing-data-model-object-proto :viscomparecorr (&optional multiple-imputation)
        (setf ct (make-container :free t :type 2 :local-menus t :show nil))
		(enable-container ct)
	(setf *spreadplot-container* ct)
  
 

  (let* (
         (data (send self :data))
         (pairwisecorrelations 
          (matrix-to-list 
           (send self :pairwisecorrelations)))
         (listwisecorrelations
          (matrix-to-list 
                     (send self :listwisecorrelations)))
         (correlations 
          (if multiple-imputation 
              (send self :multiple-imputed-parametters-list) ;todav’a por definir
              (append   (list (matrix-to-list (send self :emcorrelations)))
                        (list pairwisecorrelations)
                        (list listwisecorrelations)
                        )))

         (number-of-correlations (length pairwisecorrelations))
         (npairwise (send self :npairwise))
         (n1uniwise (send self :nvar1))
         (n2uniwise (send self :nvar2))
         (box-comparecorr nil)
         (plot-comparecorr nil)
         (variables-paired nil) ;name list for the pairs of variables
         (list-variables-corr (send self :listvariables))
         (imputed-data 
          (if multiple-imputation
              (mapcar #'(lambda (data-object)
                          (send data-object :data-matrix))
                      (send self :multiple-imputed-data-list))
              (send self :imputed-data-normal)))
        (missing-by-var (send self :missing-by-var))
        (labels (send self :labels))
        (leverages (send self :leverages))
         (variables (send self :variables))
        )

    ;;this starts the boxplots of correlations

	
    (setf box-comparecorr 
          (boxplot (list (list 1 2 3) 
                         (list 1 2 3) 
                         (list 1 2 3)) :show nil)); empieza con unos datos cualquiera
    (send box-comparecorr :title "Correlations")
    (send box-comparecorr :y-axis-label "correlations") 
    (send box-comparecorr
          :new-plot correlations)
    ;(send box-comparecorr :linked t)
    (send box-comparecorr :connect-points t)	
    (send box-comparecorr :enable-connect-points t) 
    (send box-comparecorr :variable-labels (list "ML" "Pairwise" "Listwise"  ))
    
    (send box-comparecorr :point-label (iseq  (* 3 number-of-correlations)) 
          (mapcar 'princ-to-string (combine (mapcar #'(lambda (list-correlations)
                                               (/ (round (* 100 list-correlations)) 100))
                                           correlations))))

    (defmeth box-comparecorr :plot-help ()
      (plot-help-window (strcat "Help for " (send self :title)))
      (paste-plot-help (format nil "These boxplots (see explanation of boxplots) show the values in the lowest triangle of the matrix of correlations for data  computed in different ways. From left to right you can see the distribution of the Maximum Likelihood (ML) correlations computed using the Estimation-Maximization algorithm, the correlations using pairwise deletion and the correlations using listwise deletion. These methods are designed to deal with missing values. Data with no missing values will produce the same correlations using any of the different methods. 
These boxplots are connected son you can examine each correlation one by one pointing with the mouse in each point. Points changing between boxplots are more interesting and should be checked carefully. Also, the boxplots can be compared to determine whether any of the methods produces bigger correlations than the others. 
The information in this plot is the same as in the scatterplot matrix of correlations and both plots can be used complementarily."))
      (show-plot-help))

     ;;this starts the scatterplot matrix of correlations
    

     ;;this starts the list of paired variables 
    
    (setf variables-paired (name-list (first list-variables-corr) :show nil))
    (send variables-paired :title "Variables Paired")
    ;(send variables-paired :linked t)
    (defmeth variables-paired :compute-size ()
      (send self :size 
            (round (* 1.2 (send self :text-width                                 
                                (first (first list-variables-corr)) )))
            (round (* 24 (length list-variables-corr)))
            )
      )
    (send variables-paired :compute-size)
    (defmeth variables-paired :plot-help ()
            (plot-help-window (strcat "Help for " (send self :title)))
            (paste-plot-help (format nil "This list shows all the combinations by pairs of the names of the variables in the dataset analyzed. Scroll down until finding the pair you are interested in."))
            (show-plot-help))


    ;; this starts the scatterplots of imputed data
  
          (setf plot-imputed-pairwise (plot-points
                                       (if multiple-imputation
                                           (list (first (col imputed-data 1))
                                                 (first (col imputed-data 0)))
                                           (list 
                                            (col imputed-data 1) 
                                            (col imputed-data 0)))
                                           :show nil))
          (send plot-imputed-pairwise  :title "Imputed Data")
          (send plot-imputed-pairwise :x-axis :labeled t) ;I am not very sure If I like labels
          (send plot-imputed-pairwise :y-axis :labeled t)
          (send plot-imputed-pairwise :variable-label 1 (select variables 0))
          (send plot-imputed-pairwise :variable-label 0 (select variables 1))
          (if (not multiple-imputation)
              (missing-in-plot plot-imputed-pairwise missing-by-var 1 0))   
          (send plot-imputed-pairwise :point-label (iseq (length labels)) 
                (mapcar 'princ-to-string labels))
    (send plot-imputed-pairwise :point-color (iseq (length labels)) 'blue)
          (send plot-imputed-pairwise 
                :plot-buttons 
                :margin (list 0 17 0 0) 
                :new-x nil 
                :new-y nil 
                :mouse-mode t)
    (defmeth plot-imputed-pairwise :plot-help ()
      (plot-help-window (strcat "Help for " (send self :title)))
      (paste-plot-help (format nil "This plot shows data in the pair of variables currently selected. Lines, points and symbols are added to enhance its interpretation. They have the following meaning: Points with BLACK color: data available listwise. Points with GREEN color: data available pairwise but that would be deleted listwise. Points with RED color: data actually missing in the variables currently selected and that have been estimated using the Maximum Likelihood method. There are three symbols for points with red color: X, value missing on the x variable; CROSS, value missing on the y variable; DIAMOND, value missing on both variables simultaneously.
LINES: The lines in the plot are OLS regression lines for the pair of variables under consideration. BLACK LINE: listwise regression. GREEN LINE: pairwise regression. RED LINE: regression for all the data (this includes estimated data).
This plot should help you to understand the differences among correlations computed under different methods for the same pair of variables. For example, all the red points might fall on one side of the scatterplot so the red line separates clearly of the others two. This might mean that cases with missing information on these variables are not similar to the rest of cases in the other variables, so it is predicted that they will be different on the missing data. This information can be interesting as is or as a motivation for further collection of data."))
      (show-plot-help))

    ;; this starts the plot of leverages

    (setf npplot-leverages (npplot leverages))
    (missing-in-plot npplot-leverages missing-by-var 1 0)
    (send npplot-leverages 
          :plot-buttons 
          :margin (list 0 17 0 0) 
          :new-x nil 
          :new-y nil 
          :mouse-mode t)
    (send npplot-leverages :adjust-to-data)
     (send npplot-leverages :point-color (iseq (length labels)) 'blue)
   	(defmeth npplot-leverages :plot-help ()
      (plot-help-window (strcat "Help for " (send self :title)))
      (paste-plot-help (format nil "This plot shows leverages for the dataset. The leverages are a measure of how a point separates of the center of the multivariate cloud of data. Points with high leverages have a high influence on the multivariate analysis. If a value is missing in a case with a high influence and has to be estimated the Maximum Likelihood correlations for the variable with the missing value are probably going to be quite different of the pairwise or listwise correlations. The values of leverage are plotted against their z-scores and data with high leverages will stick out on the upper-right side of the plot.
Points and symbols are added to the plot to enhance its interpretation. They have the following meaning: Points with BLACK color: data available listwise. Points with GREEN color: data available pairwise for the currently selected pair of variables but that would be deleted listwise. Points with RED color: data missing in the currently selected pair of variables and that have been estimated using the Maximum Likelihood method. There are three symbols for points with red color: X, value missing on the x variable; CROSS, value missing on the y variable; DIAMOND, value missing on both variables simultaneously."))
      (show-plot-help))

     ;; this starts the multiple boxplot

    (if (not
         multiple-imputation)
         (setf multiple-boxplot 
               (bx-id variables imputed-data labels missing-by-var)))
    
    
    
    	;; this starts the list of observations
    	  	
    (setf observations-list (name-list labels :show nil :title "Observations"))
    (send observations-list :use-color t)
    (send observations-list :point-color (iseq (length labels)) 'blue)

     ;; this starts the spreadplot
    ;(send observations-list :linked t)
   ; (send plot-imputed-pairwise :linked t)
    ;(send multiple-boxplot :linked t)
    (send variables-paired :has-h-scroll t)
    
     (setf matrix-comparecorr 
           (spread-plot (matrix (list 2 3 ) 
                                (list variables-paired
                                      box-comparecorr 
                                      plot-imputed-pairwise 
                                      observations-list
                                      multiple-boxplot
                                      npplot-leverages
                                          ))
                        :container ct
			;:supplemental-plot variables-paired
			:show t
			:rel-widths (list .5 1.5 1)
           		:span-down (matrix (list 2 3) (list 1 1 1 1 1 1))
                                         ))
    (send matrix-comparecorr :make-spreadplot-container-resize *spreadplot-container* )
    (send ct :frame-location 0 0)
 
    (apply #'send ct  :size 
           (send ct :fix-splot-size (send ct :size)))
    (send ct :show-window)
    (disable-container)
   
    (defmeth matrix-comparecorr :spreadplot-help ()
      (plot-help-window (strcat "Spreadplot Help"))
      (paste-plot-help (format nil "This spreadplot is designed to evaluate the effects of missing data on the correlation matrix. This analysis is to be regarded as a previous to any multivariate data analysis when your data have missing values. Missing values can not be used when computing the matrix of variance-covariance, so they have to be dealt with before others analysis start. The spreadplot shows the consequences of aplying three of the most common methods used to manage this situation.
¥ LISTWISE CORRELATIONS: Any case with a missing value is deleted of the dataset. That means that if a case has all the information about the variables in the study except one it will be completely deleted as if no information about it  were available at all. This procedure is quite wasteful but also is the most simple. It is widely used in many statistical packages that incorporate it as a default.
¥ PAIRWISE CORRELATIONS: Each correlation is computed separately. Only the cases with missing values in the pair of variables used for computing each correlation will not be considered. This method uses almost all the information available. However, the matrix of variance covariance so obtained may suffer a lack of consistence (matrix non singular). 
¥ MAXIMUM LIKELIHOOD ESTIMATION OF CORRELATIONS: This is a iterative method that computes the correlations repeating these two steps: 1) Estimation of the missing values using the matrix of variances-covariances and the observed data, 2) Maximization, meaning computing of the matrix of variances-covariances like if the estimated data were not missing data. Steps 1 and 2 are repeated until  convergence is reached (i.e. the estimated matrix of variances-covariances does not change).

The spreadplot shows on the upper left corner a boxplot and a matrix of scatterplots for the matrix of correlation computed using these three methods. You want to compare whether differences of importance happen between correlations computed using one method or the other. When selecting a point in these plots, boxplots and scatterplots for the data for this particular correlation are shown. Any pattern for the estimated data (points in red) is suspicious. 
In case you decide to use the estimated data go to the create data menu command and select Imputed data with random variate in the dialog box. This adds a random variate to each estimation so problems with overfitting (inflation of correlations) are avoided. This dataset so created is appropriate for exploratory data analysis."))
      (show-plot-help))
    (send 
     variables-paired
     :selection (list 0)); This is to select the first pair of variables
   
    (send matrix-comparecorr :add-slot 'x-variable 
          (select list-variables-corr 1))
    
    (send matrix-comparecorr :add-slot 'y-variable 
          (select list-variables-corr 2))

    ;this is for labels
   
    (send npplot-leverages 
          :point-label (iseq (length labels))
          (mapcar 'princ-to-string labels))

    (send npplot-leverages  
          :showing-labels nil)

    (send plot-imputed-pairwise
          :point-label (iseq (length labels)) 
          (mapcar 'princ-to-string labels))

    (send plot-imputed-pairwise 
          :showing-labels nil) ;the labels are the same in this plot and above so it is unnnecesary to see them duplicated
    


    ;;;Starts methods of for the spreadplot. This is to be improved.


    ;;Method for adding regression lines to the plot

  
        
     ;;Method for updating the plot of data imputed

    (defmeth matrix-comparecorr :update-plot-imputed ()
      (let (
            (selection (send npplot-leverages  :selection))
            (plot-leverages npplot-leverages)
            (plot-imputed plot-imputed-pairwise)
            (x (select (cols-to-update matrix-comparecorr (send box-comparecorr :selection)) 1))
            (y (select (cols-to-update matrix-comparecorr (send box-comparecorr :selection)) 0))
            ;(multiple-box (aref (send matrix-comparecorr :slot-value 'plot-matrix) 1 2))
            (variables variables)
            )

      

        ;updates the scatterplot
        (send plot-imputed :start-buffering)
        (send plot-imputed :clear )
        (send plot-imputed 
              :add-points (list 
                           (col imputed-data x)
                           (col imputed-data y))
              :draw nil
              )
        (send plot-imputed :adjust-to-data )
        (send plot-imputed :variable-label 1 
              (select variables y))                      
        (send plot-imputed :variable-label 0 
              (select variables  x))                     
        (send plot-imputed :x-axis :labeled t)
        (send plot-imputed :y-axis :labeled t)        
        (send plot-imputed :point-label (iseq (length labels)) 
              (mapcar 'princ-to-string labels))          
        (send plot-imputed :showing-labels nil) 
        (send plot-imputed :point-color
              (iseq (length labels)) 'blue)       
        (missing-in-plot plot-imputed missing-by-var x y)      
        (send plot-imputed :selection selection)
        (send plot-imputed :buffer-to-screen)
        ;updates the plot of leverages
        (send plot-leverages :start-buffering)
        (send plot-leverages :point-color (iseq (length labels)) 'blue)
        (send plot-leverages :point-symbol (iseq (length labels)) 'disk)                    
        (missing-in-plot plot-leverages  missing-by-var x y)
        (send plot-leverages :point-label (iseq (length labels))
              (mapcar 'princ-to-string labels))
        (send plot-leverages  :showing-labels nil)
        (send plot-leverages :selection selection)
        (send plot-leverages :buffer-to-screen)
        ;(send multiple-box :selection selection)
        
     ))
   


    ;;This methods keeps the selected points when new pairs of variables
    ;; are selected

    (defmeth matrix-comparecorr :update-low-row (selection)
      (let (
            (selection selection)
            )   
        (send plot-imputed-pairwise
               :selection selection)
        (send npplot-leverages
              :selection selection)        
        (send multiple-boxplot
              :selection selection)
        (send observations-list
              :selection selection)      
        ))

    ;;This propagates the selection in the low row of graphics. Basically is a link
    ;; but using messages. There must be an easier way to do this.
    ;; There are four methods. One for each plot in the lower side

    (defmeth plot-imputed-pairwise 
      :do-click (a b c d)
      (let (
            (selection  nil)
            )
        (call-next-method a b c d)               
        (setf selection (send self :selection))    
        (send matrix-comparecorr :update-low-row selection)
        
        ))

    (defmeth npplot-leverages 
      :do-click (a b c d)
      (let (
            (selection  nil)
            )
        (call-next-method a b c d)               
        (setf selection (send self :selection))    
        (send matrix-comparecorr :update-low-row selection) 
        
        ))

    (defmeth multiple-boxplot 
      :do-click (a b c d)
      (let (
            (selection  nil)
            )
        (call-next-method a b c d)               
        (setf selection (send self :selection))    
        (send matrix-comparecorr :update-low-row selection)  
        ))

    (defmeth observations-list
      :do-click (a b c d)
      (let (
            (selection  nil)
            )
        (call-next-method a b c d)               
        (setf selection (send self :selection))    
        (send matrix-comparecorr :update-low-row selection) 
        
        ))

    
    ;; This updates the boxplots and plots when
    ;; a new pair of variables is selected in the boxplots of corre-
    ;;lations
   
    (defmeth box-comparecorr
      :do-click (a b c d)  
      
      (call-next-method a b c d)
      (setf selection (send 
                       box-comparecorr :selection))
     
      (send variables-paired
            :selection selection)
     
      (defmeth plot-imputed-pairwise :redraw ()
        (let 
      (
       (plot-imputed plot-imputed-pairwise)
       (x (select (cols-to-update matrix-comparecorr (send box-comparecorr :selection)) 1))
       (y (select (cols-to-update matrix-comparecorr (send box-comparecorr :selection)) 0))
       ) 
     (send plot-imputed-pairwise :start-buffering)
      (call-next-method)  
      (lines-by-color plot-imputed 
                      missing-by-var 
                      x 
                      y
                      (col imputed-data 
                           (if x  x 1))
                      (col imputed-data (if y
                                            y 0))))
        (send plot-imputed-pairwise :buffer-to-screen))
            
      (if selection (send matrix-comparecorr :update-plot-imputed))
     
      )
        
        

    ;; This updates the boxplots and plots in the lower side when
    ;; a new pair of variables is selected in the list of names

    (defmeth variables-paired
      :do-click (a b c d)      
       
      (call-next-method a b c d)
      (setf selection (send variables-paired :selection))
      (send  box-comparecorr 
            :selection selection)

;;this method is here to avoid a bug. It is terrible but works

      (defmeth plot-imputed-pairwise :redraw ()
        (let 
      (
       (plot-imputed plot-imputed-pairwise)
       (x (select (cols-to-update matrix-comparecorr (send box-comparecorr :selection)) 1))
       (y (select (cols-to-update matrix-comparecorr (send box-comparecorr :selection)) 0))
       ) 
     (send plot-imputed-pairwise :start-buffering)
      (call-next-method)  
      (lines-by-color plot-imputed 
                      missing-by-var 
                      x 
                      y
                      (col imputed-data 
                           (if x  x 1))
                      (col imputed-data (if y
                                            y 0))))
        (send plot-imputed-pairwise :buffer-to-screen))


      (if selection (send matrix-comparecorr :update-plot-imputed))  
     	
      )

    ))



